# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
import requests
import random

# ''' 引入settings中的全局变量'''
# from scrapy.conf import settings


class MasterSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class MasterDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    def __init__(self, timeout=None, service_args=[]):
        '''初始化，设置代理池等'''
        # 设置代理池，申请10个代理IP
        proxies = requests.get('http://api.qingtingip.com/ip?app_key=f36aa4fff987f64fb7eb273ef611a6c6&num=15&ptc=http&fmt=text&lb=\r\n&port=0&mr=1&').text
        # 每个代理IP在代理池里放3个副本，每失败1次删除1个副本，即有3次失败机会。5个代理IP共15个副本。
        self.proxypool = proxies.strip("\r\n").split("\r\n") * 10
        print('ip代理池')
        print(self.proxypool)
        # pass

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~与上一个请求的分界线~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
        print('=' * 20 + '正在走process_request' + '=' * 20)
        proxy_print = random.choice(self.proxypool)
        request.meta['proxy'] = 'http://' + proxy_print
        print("正在使用代理:" + proxy_print)
        print('当前ip数量:' + str(len(self.proxypool)))

        if proxy_print in self.proxypool:
            self.proxypool.remove(proxy_print)
            print('该代理使用机会减少一次')
            print('当前ip数量:' + str(len(self.proxypool)))

        if len(self.proxypool) <= 75:
            pro_add = requests.get('http://api.qingtingip.com/ip?app_key=f36aa4fff987f64fb7eb273ef611a6c6&num=1&ptc=http&fmt=text&lb=\r\n&port=0&mr=1&').text
            print('新ip代理添加:' + pro_add)
            proxylist = pro_add.strip("\r\n").split("\r\n") * 10
            self.proxypool.extend(proxylist)  # prosylist是一个列表,proxypool也是一个列表,如果用append的话,那就是列表中含有列表的结果,extend就是把proxylist里面的值一个一个塞到proxypool里面去
        # print('ip请求407,重新请求')
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
         # 判断状态返回码,若不正常则删除本次ip并重新设定ip
        # if response.status != 200:
        #     last_proxy = response.meta['proxy']
        #     last_pro_add = request.meta.get('proxy', '').strip('http://')  # 上次请求代理IP

        #     # 判断本次失败ip是否在ip代理池中,一般都会在...
        #     if last_pro_add in self.proxypool:
        #         self.proxypool.remove(last_pro_add)  # remove() 函数用于移除列表中某个值的第一个匹配项。只删除一个
        #         print('上次请求代理:' + last_proxy + '已经从代理池中删除')

        #     # 若是三次机会都用完了,那么该条件便成立,补充ip代理池
        #     if len(proxypool) <= 12:
        #         pro_add = requests.get('http://api.qingtingip.com/ip?app_key=f36aa4fff987f64fb7eb273ef611a6c6&num=1&ptc=http&fmt=text&lb=\r\n&port=0&mr=1&').text
        #         print('新ip代理添加:' + pro_add)
        #         proxylist = proxies.strip("\r\n").split("\r\n") * 3
        #         self.proxypool.extend(proxylist)  # prosylist是一个列表,proxypool也是一个列表,如果用append的话,那就是列表中含有列表的结果,extend就是把proxylist里面的值一个一个塞到proxypool里面去
        print('=' * 20 + '正在走process_response' + '=' * 20)
        # print('网页状态码:' + str(response.status))
        if response.status == 200:
            print('=' * 20 + '成功爬取豆瓣网页' + '=' * 20)
            return response
        else:
            last_proxy = request.meta['proxy']
            print('被豆瓣服务器拒绝')
            print('=' * 20 + '重新对豆瓣网页发起请求' + '=' * 20)
            return request

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


# class MyRetryMiddleware(RetryMiddleware):
#     logger = logging.getLogger(__name__)

#     def delete_proxy(self, proxy):
#         if proxy:
#             # delete proxy from proxies pool
#             pass

#     def process_response(self, request, response, spider):
#         if request.meta.get('dont_retry', False):
#             return response
#         if response.status in self.retry_http_codes:
#             reason = response_status_message(response.status)
#             # 删除该代理
#             self.delete_proxy(request.meta.get('proxy', False))
#             time.sleep(random.randint(3, 5))
#             self.logger.warning('返回值异常, 进行重试...')
#             return self._retry(request, reason, spider) or response
#         return response

#     def process_exception(self, request, exception, spider):
#         if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
#                 and not request.meta.get('dont_retry', False):
#             # 删除该代理
#             self.delete_proxy(request.meta.get('proxy', False))
#             time.sleep(random.randint(3, 5))
#             self.logger.warning('连接异常, 进行重试...')

#             return self._retry(request, exception, spider)
